#include "DictProducer.h"
#include"cppjieba/Jieba.hpp"
#include"SplitTool.h"

#include<iostream>
#include<fstream>
#include<cstdlib>
#include<dirent.h>
#include<sys/types.h>
#include<string.h>
#include<algorithm>
#include<sstream>

using std::endl;
using std::ifstream;
using std::cout;
using std::ofstream;

//英文处理
DictProducer::DictProducer(const string & en_path){     //将所有要形成的英文文档名称写入
    //_dict.clear();
    //_index.clear();
    DIR *endir;
    struct dirent *dir;
    endir=opendir(en_path.c_str());
    if(endir==nullptr){
        cout<<"open dir error"<<endl;
    }
    while((dir=readdir(endir))!=nullptr){
        if(strcmp(dir->d_name,".")!=0&&strcmp(dir->d_name,"..")!=0){
            char path[en_path.size()];
            memset(path,0,sizeof(path));
            strcpy(path,en_path.c_str());
            strcat(path,dir->d_name);
            _files.push_back(path); 
        }
    }
    //for(auto it:_files){
    //    cout<<it<<endl;
    //}
    closedir(endir);
    //载入停用词
    ifstream stop_words;
    stop_words.open("./stop_words/stop_words_eng.txt",std::ios::in);
    string s;
    while(stop_words>>s){
        _stopWord.insert(s);
    }
    //for(auto it=_stopWord.begin();it!=_stopWord.end();it++){
    //    cout<<*it<<endl;
    //}
}

void DictProducer::buildEnDict(){       //创建英文词典
    //cout<<"DictProducer::buildEnIndex()"<<endl;
    for(auto it=_files.begin();it!=_files.end();it++){      //将文章总词一篇篇读取
        ifstream page_fp;
        page_fp.open(*it,std::ios::in);
        if(!page_fp.is_open()){
            cout<<"open error 1"<<endl;
        }
        string s;
        while(getline(page_fp,s)){
            pushDict(s);
        }
        //int i=0;
        
        page_fp.close();
    }
    for(auto it=_tmp.begin();it!=_tmp.end();it++){
            _dict.push_back({it->first,it->second});
            //cout<<i++<<" ";
    }
}

void DictProducer::buildEnIndex(){       //创建英文索引
    //cout<<"DictProducer::buildEnIndex()"<<endl;
    for(int i=0;i<26;i++){
        char ch=i+'a';
        _index.insert({ch,set<int>()});
    }
    for(size_t i=0;i<_dict.size();i++){
        for(size_t j=0;j<_dict[i].first.size();j++){
            char ch=_dict[i].first[j];
            _index[ch].insert(i);
            //cout<<i<<" ";
        }
    }
}
            
void DictProducer::storeDict(){       //将词典入文件
    ofstream en_dict;
    en_dict.open("./data/en_dictionary.txt",std::ios::out);   //打开创建英文词典
    if(!en_dict.is_open()){
        cout<<"open error"<<endl;
        return ;
    }

    for(auto it=_dict.begin();it!=_dict.end();it ++){
        en_dict<<it->first<<" "<<it->second<<endl;            //写入词典
    }
    en_dict.close();
    
    ofstream cn_dict;
    cn_dict.open("./data/cn_dictionary.txt",std::ios::out);   //打开创建中文词典
    if(!cn_dict.is_open()){
        cout<<"open error"<<endl;
        return ;
    }

    for(auto it=_CNdict.begin();it!=_CNdict.end();it ++){
        cn_dict<<it->first<<" "<<it->second<<endl;            //写入词典
    }
    cn_dict.close();

}

void DictProducer::storeIndex(){  //将中/英文索引存入
    ofstream en_index;
    en_index.open("./data/en_index.txt",std::ios::out);
    if(!en_index.is_open()){
        cout<<"open error"<<endl;
        return ;
    }

    for(auto it=_index.begin();it!=_index.end();it++){          //写入索引
        en_index<<it->first<<" ";
        for(auto s=it->second.begin();s!=it->second.end();s++){
            en_index<<*s<<" ";
        }
        en_index<<endl;
    }

    en_index.close();

    ofstream cn_index;
    cn_index.open("./data/cn_index.txt",std::ios::out);
    if(!cn_index.is_open()){
        cout<<"open error"<<endl;
        return ;
    }

    for(auto it=_CNindex.begin();it!=_CNindex.end();it++){          //写入索引
        cn_index<<it->first<<" ";
        for(auto s=it->second.begin();s!=it->second.end();s++){
            cn_index<<*s<<" ";
        }
        cn_index<<endl;
    }

    cn_index.close();

}   
//void DictProducer::getFile(){           //获取文件的绝对路径
//
//
//}                    
DictProducer::~DictProducer(){
    //cout<<"~DictProducer()"<<endl;
}

void DictProducer::pushDict(const string & word){   //存储某个单词
    string str=word;
    for(size_t i=0;i<str.size();i++){
        if(!isalpha(str[i]))str[i]=' ';
        if(isupper(str[i]))str[i]=tolower(str[i]);
    }
    //transform(str.begin(),str.end(),str.begin(),::tolower); //全部转换为小写
    std::istringstream is(str);
    string s;
    while(is>>s){
        if(_stopWord.find(s)==_stopWord.end()){
            //cout<<s<<" ";
            auto ret=_tmp.insert({s,1});
            if(!ret.second){
                ret.first->second++;
            }
        }
    }
}     
////////////////////////////////////////////////////////////////////////////////////////////////////////////
//中文处理

DictProducer::DictProducer(const string &cn_path,SplitToolCN *splitTool){        //构造函数，专为中文处理
    DIR *cndir;
    struct dirent *dir;
    cndir=opendir(cn_path.c_str());
    if(cndir==nullptr){
        cout<<"open dir error"<<endl;
    }
    while((dir=readdir(cndir))!=nullptr){
        if(strcmp(dir->d_name,".")!=0&&strcmp(dir->d_name,"..")!=0){
            char path[cn_path.size()];
            memset(path,0,sizeof(path));
            strcpy(path,cn_path.c_str());
            strcat(path,dir->d_name);
            _CNfiles.push_back(path); 
        }
    }
    closedir(cndir);
    
    //中文处理工具的初始化
    _splitTool=splitTool;

}   


void DictProducer::buildCnDic(){
    for(auto it=_CNfiles.begin();it!=_CNfiles.end();it++){      //将文章总词一篇篇读取
        ifstream page_fp;
        page_fp.open(*it,std::ios::in);
        if(!page_fp.is_open()){
            cout<<"open error 1"<<endl;
        }
        string s;
        _tmp.clear();
        while(getline(page_fp,s)){
            vector<string>CNtmp;
            CNtmp=_splitTool->cutCN(s);
            for(auto it=CNtmp.begin();it!=CNtmp.end();it++){
                auto ret=_CNtmp.insert({*it,1});
                if(!ret.second){
                    ret.first->second++;
                }
            }
        }
        //int i=0;
        page_fp.close();
    }
    for(auto it=_CNtmp.begin();it!=_CNtmp.end();it++){
            _dict.push_back({it->first,it->second});
            //cout<<i++<<" ";
        }
}              //创建中文词典
    

void DictProducer::buildCnIndex(){      //创建中文索引
    for(auto it=_CNdict.begin();it!=_CNdict.end();it++){
       string tmp=it->first;
        _CNindex.insert({tmp,set<int>()});
    }

    for(size_t i=0;i<_CNdict.size();i++){
        string tmp=_CNdict[i].first;
        _CNindex[tmp].insert(i);
    }
}
//测试
void DictProducer::start(){
    buildEnDict();
    buildEnIndex();
    storeDict();
    storeIndex();
}                   //开始    
